{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Otto Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，进行XGBoost参数调优探索。\n",
    "\n",
    "竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "#dpath = '/Users/qing/desktop/XGBoost/data/'\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"Otto_train.csv\")\n",
    "#train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "选择该数据集是因为的数据特征单一，我们可以在特征工程方面少做些工作，集中精力放在参数调优上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEFCAYAAAD0cwBnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHoFJREFUeJzt3XuYXFWZ7/FvJ50LgU5sJRE8Rw2g/kAGhAcFlASCBjAM\nxyiicrgrEy4HDXg5IBIUPCDCKArOCDMR5RK8jEFEg+EiMZBEIEeMkAzyYoIOqAFbzKUxJDFJzx9r\nlymapHrvTlft6vTv8zw81F61qtbbVZV6a+219lotXV1dmJmZ5TWo7ADMzKx/ceIwM7NCnDjMzKwQ\nJw4zMyvEicPMzAppLTuAeuvo6PS0MTOzgkaPbmvZ2n3ucZiZWSFOHGZmVogTh5mZFeLEYWZmhThx\nmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhdb1yXNJBwJURMUHSGGA60A4MBk6JiGWSpgBnAhuA\nyyJilqQdgBnAGKATODUiOiQdDFyT1b0nIi6tZ/xmZvZydUscks4HTgb+mhVdBdwaEf8h6XBgT0l/\nBaYCbwWGA/Ml3QucDSyOiEskHQ9MA84FrgfeDzwF3Clp/4hYVK+/oVGWzzyxlHZ3Pe7WUto1s/6t\nnj2OZcCxwC3Z8SHAY5J+CvyOlAjeBSyIiHXAOklLgX2BcaREAzAbuFjSSGBYRCwDkHQ3MBGomTja\n20fQ2jq4L/+uPre8pHZHj24rqWUz68/qljgi4jZJY6uKxgIrImKipM8CFwBPAquq6nQCo4CRVeXV\nZau71d29pzhWrFjTy79g+9fR0Vl2CGbWpGr9sGzk4PjzwI+y2z8mnZ5aDVRH1was7Fa+pbLqcjMz\na6BGJo75wNHZ7UOB/wQWAuMlDZc0CtgLWAIsqKo7CZgXEauB9ZL2kNQCHAXMa2D8ZmZGYxPHJ4FT\nJP0ceDfwhYh4FriWlADmABdFxFrgOmBvSfOBM4DK7KmzgFtJCWdRRDzcwPjNzAxo6eravvc56g8b\nOXlWlZk1G2/kZGZmfcaJw8zMCnHiMDOzQpw4zMysECcOMzMrxInDzMwKceIwM7NCnDjMzKwQJw4z\nMyvEicPMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCnHiMDOzQpw4zMysECcOMzMrpLWeTy7pIODK\niJhQVXYC8LGIeHt2PAU4E9gAXBYRsyTtAMwAxgCdwKkR0SHpYOCarO49EXEpZmbWUHVLHJLOB04G\n/lpVtj9wOtCSHe8CTAXeCgwH5ku6FzgbWBwRl0g6HpgGnAtcD7wfeAq4U9L+EbGoXn+DWX91+v33\nl9LuDYcdVkq71lj1PFW1DDi2ciDpVcAXgPOq6hwILIiIdRGxClgK7AuMA+7K6swGJkoaCQyLiGUR\n0QXcDUysY/xmZrYFdetxRMRtksYCSBoM3AB8AnixqtpIYFXVcScwqlt5ddnqbnV37ymO9vYRtLYO\n7t0f0SDLS2p39Oi2klq27ZU/UwNDXcc4qhwAvBG4jnRK6s2SvgrMAao/aW3ASlKCaKtRVl1e04oV\na7Y19u1WR0dn2SHYdsafqe1HrR8BDUkcEbEQ2Bsg64V8NyLOy8Y4Lpc0HBgG7AUsARYARwMLgUnA\nvIhYLWm9pD1IYxxHAR4cNzNrsFKn40bEs8C1wDxS7+OiiFhL6pnsLWk+cAabE8RZwK2khLIoIh5u\nfNRmZgNbS1dXV9kx1FVHR2fT/4HLZ55YSru7HndrKe1a/XlWlW2r0aPbWrZ2ny8ANDOzQpw4zMys\nECcOMzMrxInDzMwKceIwM7NCnDjMzKwQJw4zMyvEicPMzApx4jAzs0KcOMzMrJBciUPS0Oz/b5D0\nj5KccMzMBqgeE4CkzwLfkPQ64AHg48C/1TswMzNrTnl6Du8BpgAnADMiYiKwf12jMjOzppUncQyO\niHXAMcBPstNUO9Y3LDMza1Z5Esd9kpYAQ0mnqu4HflTXqMzMrGn1mDgi4lOk3fgOjohNwMci4oK6\nR2ZmZk0pz+B4O3Ax8FNJrwKmZmVmZjYA5TlVNR34/8CrgE5gOTCjnkGZmVnzas1RZ7eI+HdJZ0fE\neuAiSY/meXJJBwFXRsQESfsBXwM2AuuAUyLiOUlTgDOBDcBlETFL0g6k5DSGlKxOjYgOSQcD12R1\n74mIS7fQrJmZ1VGeHscGSaOALgBJbwQ29fQgSecD3wCGZ0XXkMZHJgA/AC6QtAswFTgEOAq4QtIw\n4GxgcUSMB24GpmXPcT1pWvA44CBJnhZsZtZgeRLH54C5wOsl/RCYz+Yv8lqWAcdWHR8fEb/KbrcC\na4EDgQURsS4iVgFLgX1JieGurO5sYKKkkcCwiFgWEV3A3cDEHHGYmVkf6vFUVUTcJekXwEHAYODM\niHgux+NukzS26ng5gKR3AB8FDiX1MlZVPawTGAWMrCqvLlvdre7uPcXR3j6C1tbBPVUr1fKS2h09\nuq2klm175c/UwNBj4pB0OGns4RBJAh6UdFJE/LxoY5I+BFwE/GM2ZrEaqP6ktQErSQmirUZZdXlN\nK1asKRrmgNHR0Vl2CLad8Wdq+1HrR0CeU1VfJg1eExFBuqbjmqJBSDqJ1NOYEBFPZcULgfGShmfj\nKHsBS4AFWTsAk4B5EbEaWC9pD0ktpN7KvKJxmJnZtsmTOIZHxJLKQUQ8AQwp0oikwcC1pF7CDyTN\nlXRpRDyblc8D5gAXRcRa4Dpgb0nzgTOAyuyps4BbSQlnUUQ8XCQOMzPbdnmm4z4h6Urgluz4eODJ\nPE8eEb8DDs4OX7mVOtNJ14pUl60BPrCFug9VPZ+ZmZUgT4/jdGAn4DukqbE7kVbLNTOzASjPrKoV\nwDkNiMXMzPqBPLOqTgO+BFTWp2oBuiKiuee4mplZXeQZ4/gsaSbUkh5rmpnZdi/PGMcfnDTMzKwi\nT4/jEUkzgXtIy4QAEBE31y0qMzNrWnkSxyjS8h5vryrrIs2wMjOzASbPrKoPQ9rQKZthZWZmA1ie\nWVVvAb4HjMj2w3gA+GBE/LLewZmZWfPJMzj+NeB9wPMR8UfSXhnX1zUqMzNrWnkSx4iI+HXlICLu\nBYbVLyQzM2tmeRLHX7LTVZUdAE8E/lLXqMzMrGnlmVV1NnATabXalcBvgBPrGpWZmTWtPInjiIgY\nJ2lHYHC2L4aZmQ1QeRLHR4HrI+Kv9Q7GzMyaX57E8YykOcDDwIuVwoj4fN2iMjOzppUncTxUdbul\nXoGYmVn/kCdxjK1cPW5mZpYncfyDpJ0i4oWiTy7pIODKiJgg6Q3AjaRpvUuAcyJik6QpwJnABuCy\niJglaQdgBjCGtE7WqRHRkV25fk1W956IuPTlrZqZWT3luY5jE/C0pAclzan819ODJJ0PfAMYnhVd\nDUyLiPGkU16TJe0CTAUOAY4CrpA0jDQFeHFW92ZgWvYc1wMnAOOAgyTtn/cPNTOzvpGnx3F+L597\nGXAscEt2fABwf3Z7NnAksBFYEBHrgHWSlgL7khLDVVV1L5Y0EhgWEcsAJN0NTAQW1QqivX0Era3N\nvVnh8pLaHT26raSWbXvlz9TAkCdxdPXmiSPiNkljq4paIqLyXJ2k5dpHAquq6mypvLpsdbe6u/cU\nx4oVa3oT/oDQ0dFZdgi2nfFnavtR60dAnsRRPY4whNQjmEdaJbeITVW324CVpETQ1kN5T3XNzKyB\n8uzHcXj1saTdgK/0oq1FkiZExFxgEvAzYCFwuaThpIUT9yINnC8Ajs7unwTMi4jVktZL2gN4ijQm\n4sFxM7MGyzM4/hIR8Vtgz1609UngUkkPAkOBmRHxLHAtqQczB7goItYC15HWxpoPnMHmBHEWcCsp\noSyKiId7EYeZmW2Dlq6u2kMYkr7F5nGOFlKv4PcRcVydY+sTHR2dvRqjaaTlM8tZM3LX424tpV2r\nv9Pvv7/nSnVww2GHldKu9b3Ro9u2esF3njGOuVW3u4DvAz/dxpjMzKyfynOq6gfAThFxE3AfcDD5\nEo6ZmW2H8iSOW4Fds9ud2WNu2Xp1MzPbnuXpObw+It4DkO3FMU3Sr+oblpmZNas8PY4uSftUDiTt\nCfytfiGZmVkzy9Pj+BRwr6Tfk2ZV7QycVNeozMysaeW5APCnkl4H7EPqaSzzboBmZgNXj6eqJH0Q\neCQiHgH+CjwhaXLdIzMzs6aUZ4xjGmkVWrKVaQ/AS32YmQ1YeRLH0Ih4rnIQEX/CW8iamQ1YeQbH\n50v6Dul6DoAPAg/WLyQzM2tmeRLHOcDHSNu7rictSPj1egZlZmbNK8+sqnWS7gLWAYOBuRGxvu6R\nmZlZU8ozq+pk4A5gLPB64AeSPlLnuMzMrEnlOVX1SeDAiHgeQNLlpBVzv1nHuMzM6u7J2WtLafdN\nk4aX0m5fyTOranAlaQBExJ956TawZmY2gOTpcTwq6avADdnx6cCj9QvJzMyaWZ7EMQW4hHRqahBp\nT47/05vGJA0BbiKNl2zMnnsDcCNpk6glwDkRsUnSFNJMrg3AZRExS9IOwAxgDGmJ91MjoqM3sZiZ\nWe/kmVX1InBBH7V3NNAaEe+QdARwOTAEmBYRcyVdD0zO9iWfCrwVGE66luRe4GxgcURcIul40lXt\n5/ZRbGZmlkOeMY6+9CTQKmkQMJK0aOIBQGWD5Nmk5U0OBBZExLqIWAUsBfYFxgF3datrZmYNtNUe\nh6Qd67AK7guk01RPkJZnPwY4NCK6svs7gVGkpLKq6nFbKq+U1dTePoLW1sF9EXvdLC+p3dGj27Z6\n3y03HtXASDY7+bS7S2nX+katz1QzepJyZlX1t9epu1qnquYCb5P09Yjo1ZjGFnwcuDsiLpT0WmAO\nMLTq/jZgJbA6u12rvFJW04oVa/og7O1TR0dn2SG8TDPGZPn5/cunP7xOtZJbrcSxk6QZwLslvWzS\ncUT05iLAFWzePfAvpPGNRZImRMRcYBLwM2AhcHnW7jBgL9LA+QLSOMnCrO68XsRgZmbboFbiOBI4\nHBjP5jGIbfUV4JuS5pF6Gp8BfgFMlzQU+DUwMyI2SrqWlBgGARdFxFpJ1wE3SZpPWjfrhD6Ky8zM\nctpq4oiIZ4CbJT0KPA4oq78kIjb0prGIeIG0um53h22h7nRgereyNcAHetO2mZn1jTyzqoYAvyFd\nf/Et4GlJB9U1KjMza1p5LgC8BvhQRDwMIOlg4GukKbNmZjbA5Olx7FRJGgAR8RDpojwzMxuA8iSO\nv0iaXDmQ9F7g+Rr1zcxsO5bnVNUZwAxJN5D2Gl8GnFTXqMzMrGnlWavqN8BBknYEBkVE81+5YmZm\ndZOnxwFAHZYfMTOzfqjRixyamVk/l2fP8bMaEYiZmfUPeXocH617FGZm1m/kGeN4RtIc4GHgxUph\nRHy+blGZmVnTypM4Hqq63VKvQMzMrH/IMx330mwq7h6kpc138AwrM7OBK8/g+DuBR4E7gFcDv5N0\nZL0DMzOz5pRncPwK0l7fKyNiOWkJ9H+ua1RmZta08iSOQRHxbOUgIh6vYzxmZtbk8gyO/17SMUCX\npFcA5wBP1zcsMzNrVnl6HGcCJwKvBZ4C9iMtfGhmZgNQnllVfwL+t6SRwN8i4sWeHlOLpAuB95D2\nHP86aT/zG4Eu0qytcyJik6QppKS1AbgsImZJ2gGYAYwBOoFTI6JjW+IxM7Ni8syq2kfSL0m9jWck\nzZe0R28akzQBeAdwCGmQ/bXA1cC0iBhPuk5ksqRdgKlZvaOAKyQNA84GFmd1bwam9SYOMzPrvTyn\nqq4HLoqInSNiZ+DLwDd72d5RwGLgduDHwCzgAFKvA2A2MJG0Le2CiFgXEauApcC+pNldd3Wra2Zm\nDZRncHyHiJhdOYiI2yV9tpft7Qy8HjgG2A34EWnWVld2fycwChgJrKp63JbKK2U1tbePoLV1cC/D\nbYzlJbU7enRbSS1vXTPGZPn1t/fvSdaW0m5/e52622rikPS67Oajkj4N3EAabzgRmNfL9p4HnoiI\n9UBIWks6XVXRBqwEVme3a5VXympasWJNL0Pd/nV0NN+eXM0Yk+Xn9y+f/vA61UputXoc95MGrFuA\nCaSB6oou0hhEUfOBcyVdDewK7AjcJ2lCRMwFJgE/AxYCl0saDgwD9iINnC8Ajs7un0TvE5iZmfXS\nVhNHROzW141lM6MOJX3xDyJdE/JbYLqkocCvgZkRsVHStaTEMIg0xrJW0nXATZLmA+uBE/o6RjMz\nq63HMQ5JIl230V5dHhEf6U2DEXH+FooP20K96cD0bmVrgA/0pl0zM+sbeQbHbwe+CzxW51jMzKwf\nyJM4VnrTJjMzq8iTOG6UdDlwH2lWFQAR8UDdojIzs6aVJ3FMAN5GuuK7ogt4Zz0CMjOz5pYncbw1\nIt5Y90jMzKxfyLPkyGJJ+9Y9EjMz6xfy9Dh2BxZJWk66dqIF6IqI3esamZmZNaU8ieO9dY/CzMz6\njTyJ42UX52Vu7stAzMws2TTjtw1vc9BJ+RcLyZM4Dq+6PQQYDzyAE4eZ2YCUZwfAD1cfS3ol8L26\nRWRmZk0tz6yq7l4AxvZxHGZm1k/kWeTwZ6QL/iDNqNoduLOeQZmZWfPKM8ZxSdXtLuDPEfF4fcIx\nM7Nml2cHwJcN70t6XUQ8XbeozMysaeXdAbCiC3gNaXZVc2/kbWZmdZF7B0BJOwFfBo4CptQ5LjMz\na1K5ZlVJehebN3LaJyLurV9IZmbWzGoOjkvaEbiarJfRVwlD0hjgEeAI0h4fN5JOgy0BzomITZKm\nAGdm91+W7Ve+AzADGAN0AqdGREdfxGRmZvlstceR9TIWZ4f/0IdJYwjwb8CLWdHVwLSIGE8aT5ks\naRdgKnAIKWldIWkYcDawOKt7MzCtL2IyM7P8avU47gX+BhwJPCapUr6tq+N+CbgeuDA7PoA0EA8w\nO2tvI7AgItYB6yQtBfYFxgFXVdW9uKfG2ttH0Nra3OP4y0tqd/TotpJa3rpmjMny62/v35OsLaXd\nnl6n5xoUR7Ui712txJF/xaucJJ0GdETE3ZIqiaMlIioXGHYCo4CRwKqqh26pvFJW04oVa/og8u1T\nR0dn2SG8TDPGZPn5/cunGV+n7jHVSiS1ZlX9V9+F9HcfAbokTQT2I51uGlN1fxuwElid3a5VXikz\nM7MG6s1aVb0WEYdGxGERMQH4FXAKMFvShKzKJGAesBAYL2m4pFHAXqSB8wXA0d3qmplZAzU0cWzF\nJ4FLJT0IDAVmRsSzwLWkxDAHuCgi1gLXAXtLmg+cAVxaUsxmZgNWnrWq6iLrdVS8bLOoiJgOTO9W\ntgb4QH0jMzOzWpqhx2FmZv2IE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZ\nIU4cZmZWSGlXjpvZwPL/5pezJunF415RSrvbM/c4zMysEPc4rF85f37jlyq7atz3a95/2rxbGhTJ\nS904/uRS2jVzj8PMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCmnorCpJQ4BvAmOBYcBlwOPAjUAX\naV/xcyJik6QpwJnABuCyiJglaQdgBjAG6AROjYiORv4NZmYDXaN7HCcBz0fEeODdwL8AVwPTsrIW\nYLKkXYCpwCHAUcAVkoYBZwOLs7o3A9MaHL+Z2YDX6MTxfeDi7HYLqTdxAHB/VjYbmAgcCCyIiHUR\nsQpYCuwLjAPu6lbXzMwaqKGnqiLiBQBJbcBMUo/hSxHRlVXpBEYBI4FVVQ/dUnmlrKb29hG0tg7u\nk/jrZXlJ7Y4e3VZSy1vnmPJrxrhqx1TOkiO1YnqStQ2MZLOe3rvnGhRHtSKfp4ZfOS7ptcDtwNcj\n4tuSrqq6u4306Vqd3a5VXimracWKNS8tmHlHb0PfNsdNLqfdGjo6OssO4WUcU37NGJdjyqc/xFQr\nkTT0VJWkVwP3ABdExDez4kWSJmS3JwHzgIXAeEnDJY0C9iINnC8Aju5W18zMGqjRPY7PAO3AxZIq\nYx3nAtdKGgr8GpgZERslXUtKDIOAiyJiraTrgJskzQfWAyc0OH4zswGv0WMc55ISRXeHbaHudGB6\nt7I1QONXuTMzs7/zBYBmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZ\nIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZ\nFdLoPce3maRBwNeBtwDrgH+KiKXlRmVmNnD0xx7He4HhEfF24NPAl0uOx8xsQOmPiWMccBdARDwE\nvLXccMzMBpaWrq6usmMoRNI3gNsiYnZ2/DSwe0RsKDcyM7OBoT/2OFYDbVXHg5w0zMwapz8mjgXA\n0QCSDgYWlxuOmdnA0u9mVQG3A0dI+jnQAny45HjMzAaUfjfGYWZm5eqPp6rMzKxEThxmZlaIE4eZ\nmRXSHwfHe0XS3sBVwAhgJ+AnwFzgzIg4vs5tvw/4QEScUHZMkkYBM4CRwFDgExHxYBPEtSPwbaAd\nWA+cGhF/KDOmqrb3BB4GXh0Ra8uMSVIL8HvgN1nRgxFxYckxDQauJl2MOwy4JCJmlRzTp4F3Z4ev\nAHaJiF261Snr3993s/bWASdFxLMlx/RKNn8nPA9MiYg/1XrMgOhxSHoF6c06LyIOBw4G9gHUgLav\nAa6g22tdYkyfAO6LiMOA04B/bZK4pgCPRMShpA/x+U0QE5JGkpa1WdetvKyY9gB+GRETsv+qk0ZZ\nMZ0MDImIQ4DJwBvKjikivlh5jUiJ9pTq+0t8rU4DFkfEeOB7wP9tgpg+A8yPiHHA14Av9PSAgdLj\nmAzMiYjfAETERkmnAO8AJgBI+ihwLLAj8GfgfcBY4FvABtIX/wnAWtIbPggYDpwVEb+q0fbPgR8C\nZzZJTF9h85dga/bY0uOKiK9mv1wBXgesLDum7Nf9v5P+Yd3RDK8TcADwPyT9DHgR+HhERMkxHQUs\nkXQnaYr8x5rgdSJ77mOBFRFxT7e7yoprMbBndnsk8LcmiOnNwEXZ7QXAv2yl3t8NiB4H8BrgqeqC\niHiBdEqksuLuq4CJEXEQ6Qv1bcARwEJgIvA5YBRwIKk7Nwk4h/QGblVEfA/Y0pznUmKKiJUR8aKk\nXUi/7C/sVqXM12qjpDmkL57bmyCmzwF3RsSjW7ivrJiWA1dkv0i/QHoPy45pZ1Iv4xjgStKXWNkx\nVVwIXLqF8rLieh44UtLjpN7GDU0Q06+A92S330M6TVbTQEkc/wW8trpA0m7AoQARsYn05nxH0g3A\n/wSGkN7UlaRFFT9KyuizSVn5DuDzwKb+FpOkfYD7gM9ExP3NElf2/O8ExgO3NUFMJwGnS5oL7AJU\n/2otK6ZfZPWIiPnAa7KeUZkxPQ/Mioiu7PP0piZ4nZD0ZmDlVrZdKCuuzwFXRcSbgSNpjs/5FcBY\nSQ+Qei/P1KgLDJzEMQt4t6Q9ACQNIQ3m/Tk73hd4b0R8iPRrdxCpyz0ZmBcR7wK+D1xA6jIuj4gj\ngcvIcT6wmWLK/jF9HzihslBkk8R1oaSTs8MXgI1lxxQRb6g6T/4s6R96qTGRvnjOy9p4C/BMRFR6\ntGXFNJ/NywC9BXi6CV4nSL/At/QZLzOuFcCq7PafSKeryo7pUGB6Nr64lJRwahowV45LOgD4Z9KL\n3Qb8GLifNPbwEdKbNiyrvo6UxR8CbiJl+cHAx0m/Cr5LyvStwOe3cP60e9sTSOcYj+9W3vCYJN1B\n2gTrd1nRqoiY3ARxvTp7/PDs8Z+OiAVV95f2/mXt/w7YM146q6qM16mddHpqJ9Ivy3Mi4omSYxoG\nXEc6V94CnB0RvywzpqzdfwXujYgfbuX+Ml6r1wDfIL1/Q4DPRsS9Jcf0BuDm7PAPwOkRsXpLdSsG\nTOIwM7O+MVBmVdWVpB8Ar+xW/LJf8o3UjDFBc8blmPJxTPk1Y1x9GZN7HGZmVshAGRw3M7M+4sRh\nZmaFOHGYmVkhThxmfUDSKElbnPbZh218S9Lr69mGWR5OHGZ9ox3Yr85tHE66TsKsVJ5VZdYHJP2I\ntIz3ncDjwLtIUx//DBwbEc9K6gAeIS1f8jbSUhDHZXWWAz+KiBuzhe3OI/2we4S01tB5Wf2lwPiI\neL6Bf57ZS7jHYdY3pgJ/JC1ctyfwjoh4E+mL/sSszs7AFyNiP1KSGQfsTVquY3/4+34MU7LH70da\nluJTEfHF7PmPdtKwsvkCQLM+FBFLJX0S+CdJAt4OLKuq8nD2/yOA/4iI9cD6qvGRw4E3Ag+lhzMU\n+CVmTcSJw6wPZWsNfYe0ON1M0mKNfx+XiIgXs5sb2XKPfzApoUzNnm8n/O/UmoxPVZn1jQ2kL/jD\ngLkRcT1prONIUjLo7l7g/ZKGKu0yeAxp35a5wPskjcmWS7+ObDXcqjbMSuXEYdY3niMtJ/6/gLdI\negyYAzwG7Na9ckT8BHgAWEQaUP8j8GK2adSl2WP/k/Rv9IvZw2YBP8n2aDArjWdVmZVA0tuBN0XE\nTdm+Cw8CH4mIx0oOzaxHThxmJZD0SuDbwK6kXsVNEfGlcqMyy8eJw8zMCvEYh5mZFeLEYWZmhThx\nmJlZIU4cZmZWiBOHmZkV8t8KhP35E+3DFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10f52c310>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.target);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每类样本分布不是很均匀，所以交叉验证时也考虑各类样本按比例抽取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['target']\n",
    "y_train = y_train.map(lambda s: s[6:])\n",
    "y_train = y_train.map(lambda s: int(s)-1)\n",
    "\n",
    "train = train.drop([\"id\", \"target\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "再次调整弱分类器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 9\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print cvresult\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds4_2_3_699.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators4_2_3_699.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     test-mlogloss-mean  test-mlogloss-std  train-mlogloss-mean  \\\n",
      "0              1.987845           0.005728             1.985574   \n",
      "1              1.839574           0.009242             1.835558   \n",
      "2              1.713490           0.009917             1.707658   \n",
      "3              1.608165           0.007763             1.600937   \n",
      "4              1.518129           0.006750             1.509076   \n",
      "5              1.441628           0.007346             1.431055   \n",
      "6              1.373809           0.009413             1.361750   \n",
      "7              1.315353           0.010863             1.301592   \n",
      "8              1.261068           0.008643             1.246300   \n",
      "9              1.212350           0.007709             1.196243   \n",
      "10             1.167628           0.006614             1.150325   \n",
      "11             1.128763           0.007499             1.110312   \n",
      "12             1.092276           0.008000             1.072766   \n",
      "13             1.059096           0.007159             1.038498   \n",
      "14             1.028031           0.006123             1.006339   \n",
      "15             0.999565           0.005832             0.976816   \n",
      "16             0.974108           0.006457             0.950437   \n",
      "17             0.949175           0.005982             0.924712   \n",
      "18             0.926616           0.005607             0.901288   \n",
      "19             0.906225           0.005026             0.880014   \n",
      "20             0.887363           0.004047             0.860339   \n",
      "21             0.869405           0.004806             0.841648   \n",
      "22             0.853281           0.005358             0.824585   \n",
      "23             0.837299           0.005076             0.807780   \n",
      "24             0.823196           0.004454             0.792966   \n",
      "25             0.809317           0.003919             0.778280   \n",
      "26             0.796771           0.004103             0.764896   \n",
      "27             0.785107           0.004328             0.752517   \n",
      "28             0.774038           0.004470             0.740719   \n",
      "29             0.763417           0.004355             0.729318   \n",
      "..                  ...                ...                  ...   \n",
      "669            0.483747           0.007161             0.223055   \n",
      "670            0.483838           0.007159             0.222864   \n",
      "671            0.483836           0.007173             0.222672   \n",
      "672            0.483880           0.007178             0.222463   \n",
      "673            0.483859           0.007242             0.222208   \n",
      "674            0.483887           0.007229             0.222050   \n",
      "675            0.483870           0.007218             0.221836   \n",
      "676            0.483865           0.007251             0.221629   \n",
      "677            0.483756           0.007269             0.221432   \n",
      "678            0.483677           0.007261             0.221235   \n",
      "679            0.483672           0.007273             0.221034   \n",
      "680            0.483675           0.007190             0.220844   \n",
      "681            0.483702           0.007262             0.220620   \n",
      "682            0.483704           0.007196             0.220416   \n",
      "683            0.483802           0.007152             0.220215   \n",
      "684            0.483860           0.007226             0.220016   \n",
      "685            0.483838           0.007168             0.219794   \n",
      "686            0.483904           0.007200             0.219584   \n",
      "687            0.483910           0.007150             0.219374   \n",
      "688            0.483979           0.007152             0.219160   \n",
      "689            0.484005           0.007189             0.218944   \n",
      "690            0.484083           0.007188             0.218793   \n",
      "691            0.484080           0.007147             0.218587   \n",
      "692            0.484115           0.007113             0.218371   \n",
      "693            0.484165           0.007090             0.218190   \n",
      "694            0.484216           0.007074             0.217983   \n",
      "695            0.484290           0.007134             0.217783   \n",
      "696            0.484292           0.007194             0.217598   \n",
      "697            0.484289           0.007233             0.217413   \n",
      "698            0.484312           0.007273             0.217209   \n",
      "\n",
      "     train-mlogloss-std  \n",
      "0              0.004828  \n",
      "1              0.008462  \n",
      "2              0.009008  \n",
      "3              0.006404  \n",
      "4              0.004304  \n",
      "5              0.005693  \n",
      "6              0.007829  \n",
      "7              0.009434  \n",
      "8              0.007500  \n",
      "9              0.007431  \n",
      "10             0.006189  \n",
      "11             0.005928  \n",
      "12             0.006506  \n",
      "13             0.006500  \n",
      "14             0.005621  \n",
      "15             0.005777  \n",
      "16             0.005716  \n",
      "17             0.005255  \n",
      "18             0.005120  \n",
      "19             0.003845  \n",
      "20             0.003855  \n",
      "21             0.003999  \n",
      "22             0.004093  \n",
      "23             0.004174  \n",
      "24             0.004180  \n",
      "25             0.004257  \n",
      "26             0.003435  \n",
      "27             0.003070  \n",
      "28             0.002743  \n",
      "29             0.002891  \n",
      "..                  ...  \n",
      "669            0.002918  \n",
      "670            0.002956  \n",
      "671            0.002946  \n",
      "672            0.002951  \n",
      "673            0.002960  \n",
      "674            0.002946  \n",
      "675            0.002956  \n",
      "676            0.002943  \n",
      "677            0.002976  \n",
      "678            0.002992  \n",
      "679            0.002994  \n",
      "680            0.002996  \n",
      "681            0.002945  \n",
      "682            0.002955  \n",
      "683            0.002957  \n",
      "684            0.002966  \n",
      "685            0.002956  \n",
      "686            0.002963  \n",
      "687            0.002957  \n",
      "688            0.002940  \n",
      "689            0.002914  \n",
      "690            0.002887  \n",
      "691            0.002898  \n",
      "692            0.002904  \n",
      "693            0.002915  \n",
      "694            0.002902  \n",
      "695            0.002895  \n",
      "696            0.002894  \n",
      "697            0.002923  \n",
      "698            0.002912  \n",
      "\n",
      "[699 rows x 4 columns]\n",
      "logloss of train :\n",
      "0.244918398432\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAETCAYAAADZHBoWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8XGW9+PHPmSWZmWxNk7SllJ3yZS9QlhYQqoKKgiB6\nF6teBRG5XldU1F5cr6goyCJwRVbFBUEuV5CL4A8QLauALGX51kILpWuaZp9kMtvvj+dkOk2zNc1k\nZjrf9+sVcvbzPUNzvvM8zznP42WzWYwxxhiAQLEDMMYYUzosKRhjjMmxpGCMMSbHkoIxxpgcSwrG\nGGNyLCkYY4zJCRU7ADM1ROQI4CHg7ar6lL+sGXgC+Kyq3uMv+zhwLlAPVAGvAReq6hP++j8DewCd\ngOdv8xtV/U4BYj4K+LiqnjfZx96OGM4BqlT1GhE5D5imqj+YpGPfDyxW1U2TcbzJIiIfAz6gqqcW\n4Nh7AstUtXayj20mhyWFCqGqz4jIBcDtfoLoBm4DbsxLCN8DTgD+WVVf95e9DfiDiMxX1Tf8w31Z\nVX/nr58GvCQiD6jqI5Mc9kHAnEk+5vY6HlgGoKo/neRjnzzJxzNmh1lSqCCqeq2IHA/cCLwKtAPf\nAxCRmcDngX1UdV3ePg+KyPlAzQiHrfN/b/KPcxBwFdAEZIFLVfUX/rpzgc8CaWAD8GlVXe7H9GMg\n6O/zfeBJ4DtAg4jcpKpn5Z9URFYBNwNvB3YHfquqF4x2/SJSBVwMnOif6++4UlKXiPw7cB4wAPQD\nnwQEeC9wsoj0AS1As6p+2j//r4H3+Nf6TeA4YD6QBN6rqmtF5FRgCa5ENQP4uap+XURu8sN6SETe\njSuZbfO5icgi4AqgF/f/4ATgemAukAGeBj6pqpm863yHv/8h/vw0YCWwN/CvQ69TVV8a7XMb8hmO\n9v/3q8DHcV84/gKcoap7bsexG4CrgcP8Y98LLFHVlIh8G3ifH3cb8DFVXTfS8vGe02zL2hQqz3nA\nwcAHgI+q6uAr7QuBl4f7g1LVW1T15bxFPxKRZ0XkRVxy+X/AchEJAXcBP1HVQ4FTgO+JyEK/xHEB\n8FZVnYe7of6viHjAt4Efq+p84Gzgbaq6GvgG8NehCSFPraq+BTgW+IyI7DXGtX8VSAHz/RjWAj8Q\nkSBwOfAuVT0K+BlwvKre6V/PZap69TDHi/jH+aK/zxX+/GrgY/61fRH3OR8JLAC+JiLNedf0VmDd\nSJ+bv83BwAf9Y78XqFPVw4Cj/PV7D4nrT0CtiBzpz38QuAfoGu46x/jMcsb4//tO4GN+TPPZ8mVh\ne1yJu7EfAhwJzAO+JCK74b6wHOV/jvcDx4y0fALnNXksKVQewf3BTsP98Q7ycN/O3EYidf6N/1kR\nWeFXLQ36sqoepqoHATOBPXE33P1wN8r/AVDVtcAdwLv8n9+qaqu/7mZgV3/f24CrReRXfkxLxnkt\nv/ePtQbYCEwfY/tTgdOBv4vIs8AZwIGqmgZuBx4Vkatw7SU3jOP8d/i/XwXWq+pzefPT/YR7GjBf\nRL6JKw15bFvqGu1zA1g9WJ0HLAUO8tt2vgpcrqor8g/mn/cG3E0a4Czg+h24zvHE+W7gdlXt8M8/\nXBIdyynAVaqaVdUE8FN/2RrgOeAZEbkEeFZV/3eU5WYHWFKoIH7D8v8AX/B/bhWRWf7qJ4D9RaQJ\nQFW7/Rv/YcAvcdUb21DVduBWXLXGcP+eAkB4hHUeEFbVa3HfDv8EvBN43q9KGEtf3nTWP95ogsDn\n8q7raFyJCVX9MO4GvgL4Cu5zGksibzo5dKWI1OCqqI4AngG+7G83NM7RPjeAnsGFqroS2BdXxVYP\n/D8R+cAw+98E/LOIHIZrHP+zv/9ErnM8cabY+rrS23HckY4fwP37yOCq/D6GK0lcJiJXjLR8Auc1\neSwpVAi/iuQ24G5V/Y2q3gTch0sMQf9b3xW4hujd8/bbHVdXPuwfuYiEcd/AnwQUGBCRM/11s4H3\n42729wH/IiIt/rqzcH/IK0TkUeBwv/RwLq4U04i70YSZPPcBnxaRKhEJANcB3xeRZhFZDbSp6uXA\nhbiqC3Ywhrm4G/eFqno37gZWjUtO4D7TMKN/blvx2z5uAu5X1a/413Tw0O380tMTwLW4NgjGuM7x\nGC3Oe4D35yXzj5NX8hyn+4D/EBFPRKpx/xb+JCLzcI39L6vq94HLgHkjLd/Oc5ohrKG5cvwIiOHq\nuAd9Cnfj+B7wFVX9TxH5EPArEanF3bD6gd+ydXXAj0TkQtwffQ3wAHCRqiZF5AzgShH5Fu7f13dU\n9SEAEbkMeNC/IbcCp6pqxn8q6goR+S6u8fTbqrrKT2QXicidqvq+SfgM/gu4BPftPQg8C3zRb2j+\nLvCA36CcAs7x97kXuEpEJnK+54E/AK+ISAfu2/lLuG/6r+K+pS/FVWkN+7n5Dc35fgEswj3x1Qu8\ngUvmw7kO+B2uHQJV3TTKdQ71LhHpyZvvUNU5Y/z/vQ54TETiwItAfIRj1ww5Nrg2rc8CPwFewDXM\n/xH372pARG4DnvL368M9IPDccMtHOKcZJ8+6zjbG7Ci/UftYVb3Snz8fOEZV/6W4kZntZSUFs9MQ\n93X+tyOsVrtBFdRy4Cv+Y8dZXAnm3OKGZCbCSgrGGGNyrKHZGGNMjiUFY4wxOWXfptDa2j3h+q/G\nxhjt7SM9IFF6yinecooVyivecooVLN5C2pFYW1rqhn2vp6JLCqFQcOyNSkg5xVtOsUJ5xVtOsYLF\nW0iFiLWik4IxxpitWVIwxhiTY0nBGGNMjiUFY4wxOZYUjDHG5BTkkVS/58wbcX3lVwPfVdW78taf\nhhtAJYUbDvI6v5O0a3C9HCaAc4b2E2+MMaawClVS+DCue9634AbguGpwhZ8wLgPegetK+FxxQ0Ge\ngRvAYyFu8JBLCxSbMcaYERTq5bXbcV32ght4I5W37gBghT84CyKyFDdAy0JcV7mo6uN5QwmOqrEx\nNqFndbPZLNlslpaWiYwaWDzlFG85xQrlFW85xQoWbyFNdqwFSQqq2gNuSEdccrgwb3U9bhjAQd1A\nwzDL0yISUtX8hLKNib7N9+8/vwlCSf77Q+XTkWNLSx2trd3FDmNcyilWKK94yylWsHgLaUdiHSmZ\nFKybC39Q7TuBa1T113mruth6UO86oGOY5YGxEsKO8Fpeh9BAoQ5vjDFlqVANzTOB+4FPq+oDQ1a/\nDMwVkem4sWdPwI2GNTjI+W0isgA3+lLBhANVJL3E2BsaY0wFKVRJYQlujN2vi8jX/WXXATWq+jN/\nVKb7cA3dN6rqGhG5EzjZH6/XA84qUGwABAiClynkKYwxpuwUqk3hc8DnRll/N3D3kGUZ4LxCxDOc\nAEG8QJZkOkU4WPadxRpjzKSo2JfXArgnlvoHCtZsYYwxZaeCk4K79P5UssiRGGNM6ajcpOC5KqOB\nlD2BZIwxgyo2KQQ9v/rISgrGGJNT8UkhYUnBGGNyKj4pDCStodkYYwZVcFJwbQr9aSspGGPMoIpN\nCiG/pJBMWUnBGGMGVWxSCAb8NgUrKRhjTE7FJoVwwH8k1ZKCMcbkVGxSCPlJwaqPjDFmi4pNCi+t\ndEM3WEnBGGO2qNikkKldD8BAxkoKxhgzqGKTQlXYf/oobUnBGGMGVWxSOG7XowFLCsYYk69ik0JV\nMAxAMmNtCsYYM6iCk4L/9JE1NBtjTE7FJoXq0GBJwaqPjDFmUMUmhUioCoBU1pKCMcYMqtikYCUF\nY4zZVkFHrBeRY4CLVXVR3rJZwK15mx0GfFVVfyoizwBd/vKVqnpWoWKLhF1JIW1JwRhjcgqWFETk\nAuAjQG/+clVdDyzyt1kIXARcJyIRwMtPIIUUDVv1kTHGDFXIksKrwJnALcOtFBEP+AnwIVVNi8iR\nQExE7vfjWqKqj491ksbGGKFQcLuDi3t9AGQDGVpa6rZ7/2KxWAunnOItp1jB4i2kyY61YElBVe8Q\nkT1H2eQ04EVVVX8+DlwCXA/MBe4VEVHVUb/Kt7fHJxRfX497FDWZTtLa2j2hY0y1lpY6i7VAyine\ncooVLN5C2pFYR0omBW1TGMOHgSvy5pcDK1Q1CywXkTZgF2B1IU4erXLVR5lsuhCHN8aYslTMp4+O\nBB7Nmz8buBRARGYD9cC6Qp188Omj3kSiUKcwxpiyM2VJQUQWi8i5/nQL0OWXCgbdAEwTkaXAb4Gz\nx6o62hHBQJBsxoOQvdFsjDGDClp9pKqrgAX+9K/zlrfiHkXN33YAWFzIeLaRDeB52bG3M8aYClHM\nNoWiC2RDFfz6njHGbKuib4leNkQ2YO8pGGPMoIpOCgFCZD17+sgYYwZVdFIIEoJAmmzW2hWMMQYq\nPSl4IbxAhv6kVSEZYwxUeFIIeYPvKvQXORJjjCkNlhSwF9iMMWZQRSeFsD9OczxpScEYY6DSk0LA\n9X/UO2DVR8YYAxWeFN5c73pY7RsYKHIkxhhTGio6KWSqOgHoS1pJwRhjoMKTQjjsLt/aFIwxxqno\npHDUrvMASCSt+sgYY6DCk0IkXA1Af9qSgjHGQIUnhaifFBIpG1PBGGOg0pNClZ8UrKRgjDFAhSeF\nGj8pDFhSMMYYoMKTQmwwKWSs+sgYY6DCk0JNdQSApCUFY4wBCjwcp4gcA1ysqouGLP8CcA7Q6i/6\nJPAP4BpgHpAAzlHVFYWMry4SBWBde1chT2OMMWWjYElBRC4APgL0DrN6PvBvqvp03vZnAhFVXSgi\nC4BLgdMLFR9AXdQlBQI2+poxxkBhq49eBc4cYd184GsislREvuYvOx74I4CqPg4cWcDYAGiIxgDw\nwn2FPpUxxpSFgpUUVPUOEdlzhNW3AlcDXcCdInIqUA905m2TFpGQqo46LFpjY4xQKDihGNMZV0KI\nVIVpaamb0DGmWrnECeUVK5RXvOUUK1i8hTTZsRa0TWE4IuIBl6tqpz9/D3A4LkHkX11grIQA0N4e\nn3AsLS11kAmSzA7Q2to94eNMlZaWurKIE8orViiveMspVrB4C2lHYh0pmRTj6aN6YJmI1PoJ4m3A\n08AjwLsB/DaFF6YiGC8TIuPZ00fGGANTWFIQkcVArar+TESWAA/hnjJ6QFX/T0QCwMki8ijgAWdN\nRVyBbJhUwJKCMcZAgZOCqq4CFvjTv85bfgtwy5BtM8B5hYxnOEHCpAJ9pDMZgoGKfm3DGGMq++U1\ngBBVeME08YSVFowxpuKTQm9vFoDOvok3WBtjzM6i4pOCF3YlhK64JQVjjKn4pLBr7S4AdCUsKRhj\nTMUnheqg6ym1O2FvNRtjTMUnhWjI9ZTaM2BJwRhjLCn4SSFuScEYYywpxKpcUugd6C9yJMYYU3wV\nnxRqwq777OdeW1/kSIwxpvgqPikMDrSTDY7Z954xxuz0Kj4pTIvWAODFOsfY0hhjdn4VnxSaalz3\nsXXVNUWOxBhjiq/ik8K0aC0AA5lEkSMxxpjiq/ikEAtHIQsp7OkjY4yp+KQQ8AJk02GSWEnBGGMq\nPikAkArjhZJkMtliR2KMMUVlSQHIZoFQkp6+gWKHYowxRWVJAQgQwgtk6IhbVxfGmMpmSQGYEZ0B\nQFtPd5EjMcaY4hozKYjIdBE5yZ/+mojcLiIHFj60qRMLubeaN/dZUjDGVLbQOLb5DXC3iAD8E3AZ\n8FPghLF2FJFjgItVddGQ5R8EPg+kgBeAT6lqRkSeAbr8zVaq6lnjvI4dUhOOQQo6+3qm4nTGGFOy\nxlN91KiqVwGnAzer6i1AbKydROQC4HogMmR5FPgu8FZVPQ5oAE4VkQjgqeoi/2dKEgJAbZV7m/mB\n51ZO1SmNMaYkjScpBERkPnAG8AcROYzxlTBeBc4cZnkCOFZVB8e/DAH9wDwgJiL3i8iDIrJgHOeY\nFA0RlxRSdWum6pTGGFOSxnNz/wrwI+ASVX1NRB4HvjDWTqp6h4jsOczyDLABQEQ+A9QCfwIOBi7B\nlS7mAveKiKjqqN2XNjbGCIWC47iM4bW01DGnpRk2QDjk0dJSN+FjTYVSjy9fOcUK5RVvOcUKFm8h\nTXasYyYFVX1ARJaqakJE9gX+C3h4R04qIgHgh8B+wPtVNSsiy4EVqpoFlotIG7ALsHq0Y7W3x0db\nPaqWljpaW7sJZ8IATI9Mp7W1dBubB+MtB+UUK5RXvOUUK1i8hbQjsY6UTMbz9NHXgetEZHfgL7gG\n4msnFMUW1+LaGs7Iq0Y6G7jUP+dsoB5Yt4PnGZdmv6fURMbeUzDGVLbxVB+dDhyHqzL6papeICJP\nbe+JRGQxrqroKeDjwF+BB/2nmq4AbgBuFpGlQBY4e6yqo8lSH3E9pXb1907F6YwxpmSNJykE/aqj\nU4EL/aqfcQ0+oKqrgAX+9K/zVo1UQlk8nuNOtsH3FAhaNxfGmMo2nqePHhCRZUAVrvroYeCugkY1\nxVxPqUEIJclkrVM8Y0zlGjMpqOqXgHcDC/wnhz6jql8peGRTLR3CCw/Q05csdiTGGFM042lobsE9\nKrpRRDqAb4rIzIJHNsXqQ9MgNEBHtw22Y4ypXOOpProWeBLYG9gTeAzXKLxT6e4K4Hmwoauj2KEY\nY0zRjKeheW9VzX8z+Yci8pFCBVQ0qWoANvS0A7sXNxZjjCmS8ZQUsiKy2+CM/77CTlfxHqp3JYS2\neNcYWxpjzM5rPCWFrwOPicgTgAccA3yyoFEVQTDgkQI6+i0pGGMq13iePvoDcDhwI3ATcLi/bKfy\ngf1PAeCVteuLHIkxxhTPeEoKqGorcM/gvIi8oKqHFCyqIphZ1whANpQociTGGFM8Ex2Oc8/JDKIU\nNEbrAaiOTknPGsYYU5ImmhR2utd+66r8TvHotbeajTEVa6JJYacTDoTIJqvwqhJ09VofSMaYyjRi\nm4KIZBi+ROCNsLz8JaN4kW42dfYxrba62NEYY8yUGzEpqGrFlSJm1zexLtXJuo4O9t11WrHDMcaY\nKVdxN/7RbIxvAmBt16YiR2KMMcVhSSHP8XOOAuDhF18tciTGGFMclhTy7FLfBEA2ZMNyGmMq05gv\nr4nIN4YsygJ9wMuqes8wu5StXepcUqhtSBc5EmOMKY7xlBT2BU4BOvyfk4ATgU+IyA8LGNuUmx51\njcvdaes+2xhTmcaTFARYpKpXquqVwMlAs6qeAbyzoNFNsYaqerJZIDRAYsBKC8aYyjOevo8a/e0G\nOwWqAmr96VGTiogcA1ysqouGLD8N+AaQAm5U1etEJABcA8zzz3WOqq4Y53VMimAgSDUx+qv72dAe\nZ/eZdVN5emOMKbrxJIWrgKdE5A9AEFeV9BMR+Tzw/Eg7icgFwEeA3iHLw8BlwFH+ukdE5C7gOCCi\nqgtFZAFwKXD69l/SjkmmMnjhBGvbeiwpGGMqzni6zr4S+GdgLbAS+ICqXoPrNfWsUXZ9FThzmOUH\nACtUtV1VB4ClwAnA8cAf/XM+Dhy5HdcxaQJeEC+Q5ZYHnyvG6Y0xpqjG8/SRh7thH48rKQRE5GVV\n/cdo+6nqHSKy5zCr6oHOvPluoGGY5WkRCanqqN2WNjbGCIWCY13GiFpati4NnDT3WO5b+ScOkOpt\n1pWCUoxpJOUUK5RXvOUUK1i8hTTZsY6n+uiHwFzcIDsernSwF/D5CZ6zC8i/ijrcU01DlwfGSggA\n7e3xCYbhPszW1u6tls2ong7Ams4N26wrtuHiLVXlFCuUV7zlFCtYvIW0I7GOlEzGkxTegRttLQMg\nIvcAL0woCudlYK6ITAd6cFVHl+DefzgNuM1vU9iRc0zYzJoWADYnW8lms3ieV4wwjDGmKMbzSGqI\nrZNHCNju5zVFZLGInKuqSeB84D7gMdzTR2uAO4F+EXkU1xD9he09x2SYEWsGwKvqp7svWYwQjDGm\naMZTUvgV8GcR+Y0//0HgN6Nsn6Oqq4AF/vSv85bfDdw9ZNsMcN54jltINeEYZIJ4kTgbNsepj1UV\nOyRjjJky43n66HvAfwG744bhvEhVLypwXEWVTYXwquOsLpN6RWOMmSzjKSmgqvcC9w7Oi8g1qvqp\ngkVVZAc2z+XlrmW81rqet7FbscMxxpgpM9FeUj88qVGUmJU9ruvsv61cWeRIjDFmak00KezUj+SE\ngu5jCcV6yWZ3zpFHjTFmOBNNCjv1nfL8I/4dgFR1J5s6+4scjTHGTJ0R2xRE5CGGv/l7QLRgEZWA\nllgz2YxHINrNynVdtEzbqS/XGGNyRmto/tZUBVFqAl6AWdFdWM86/rGmnaMPmFnskIwxZkqMmBRU\n9eGpDKTUtA+04QWyPPyS8qGT9i92OMYYMyVsjOYRnD73XQB4kW6SKRtwxxhTGSwpjGBO7WwAstEu\nXl/fU+RojDFmalhSGMGutbtAFgI1nVx+u42tYIypDJYURhANRZgZm0mgpot95pRP3+rGGLMjLCmM\nomOgHS+YZtm6VaTSmWKHY4wxBWdJYRT/tN8ZAARqO3h1TecYWxtjTPmzpDCKvRv2ACBQ18GVv3u+\nyNEYY0zhWVIYxYxoMx4egdp2BlJWfWSM2flZUhiF53kc2nwggep+MuFe2qwfJGPMTs6Swhj2n74f\nAIGGTTylG4scjTHGFJYlhTEc2OSSQrB+E3c8/GqRozHGmMKypDCG5mgTAS9AoKGNVCZNa0dfsUMy\nxpiCGddwnBMhIgHgGmAekADOUdUV/rpZwK15mx8GfFVVfyoizwBd/vKVqnpWoWIcr3AgTCaYIFDb\nwbdufJKrzz+x2CEZY0xBFCwpAGcAEVVdKCILgEuB0wFUdT2wCEBEFgIXAdeJSATwVHVRAePabucc\n/GGufu4Gqls2UrVxFulMhmDAClnGmJ1PIZPC8cAfAVT1cRE5cugGIuIBPwE+pKppf5uYiNzvx7ZE\nVR8f7SSNjTFCoeCEg2xpGbsLi+OaDucXL9eQaN5I52sJVm7sZeEhsyd8zh0xnnhLRTnFCuUVbznF\nChZvIU12rIVMCvVA/mvAaREJqWoqb9lpwIuqqv58HLgEuB6YC9wrIjJkn620t8cnHGBLSx2trd3j\n2nYgnWSAAQJ1m/nhL57i2i8vwvOmdqjq7Ym32MopViiveMspVrB4C2lHYh0pmRSyDqQLyD9rYJib\n+4eBn+XNLwd+qapZVV0OtAG7FDDGcTv3kI8CEJq1ilQmy4srNxc5ImOMmXyFTAqPAO8G8NsUXhhm\nmyOBR/Pmz8a1PSAis3GljXUFjHHc5k7bm9pwDXXNveBluOJ3z5PJDjeEtTHGlK9CJoU7gX4ReRS4\nDPiCiCwWkXMBRKQF6FLV/DvrDcA0EVkK/BY4e7Sqo6kUDAQ5etYRxNNxqps3ks5kefzF9cUOyxhj\nJlXB2hRUNQOcN2TxK3nrW3GPoubvMwAsLlRMO+otuy7kwdV/ZY+D21j+0CxuuOdljtivhUhVIZtm\njDFm6thzldthRqyZA5uE1b2rOXFhjGwWzr/qkWKHZYwxk8aSwnZ6225vASA+7RWCAY/+gTTPv9pW\n5KiMMWZyWFLYTvs3ziXkBVnW9hKf+OddAbjid8+xuct6UDXGlD9LCtvJ8zzOm+d63vj1qp+z+KS5\nZLPwlZ8+Rl+iJNrEjTFmwiwpTIArLYRIZpLstneC6nCQdCbL565cSiKZLnZ4xhgzYZYUJsDzPM6f\n/+8AXPnsz7jyc8cRDgVIpTN85vK/WonBGFO2LClM0B71u1EdrALggTcf5uovnECVnxg+d+Vf6eod\nKHKExhiz/Swp7IDvHrsEgLtfu4+18bVcff4JVIUDpNJZzr9qKa+vL4/+U4wxZpAlhR0QC8f49GHn\nAPCjp64imUny3+efyPtO2JtMFr5989944Ok3rTsMY0zZsKSwgw6Yvh/VwWoy2QxfXfodsmQ57dg9\n+ewHDgXgV39aznmX/JlX13aOcSRjjCk+SwqT4Edv+RahgHsa6ct/+RYAh+3bzGWfHmyAznLRL57m\nk5f8mTc2WJWSMaZ0WVKYBMFAkO8ddyEA/el+/vym6/qiobaaa7+0iK8sPpxQ0COZyvCtm/7Gj297\nlmeWt5LOZIoZtjHGbMN6cpskNeEY31zwZb79+I+4ffnvCXpB3rLrAgBk90au/dIiXly1mbseWcWy\n1zaz7LXNeB6cunBPTpg3m6aGSJGvwBhjLClMqhmxFi485otc9MSPuVX/h2QmmesryfM8Dt6riYP3\nauLNjT08/OxaHnjmTe5+dBV3P7qKuXMamC8zOHSfJmY2Rqd8VDdjjAHwsmX+ZExra/eEL6BQw+6t\n7VnPRU/+GICTd1/Ee/d5FwFv25q6xECaJ1/ZwC33Kan01pdx/KG7sP/u09hvzjSaGiJ4nlcxwwQW\nQznFW06xgsVbSDs4HOew3zwtKRTof/6mvs1887EfABAKhPjOwq/RUD3yANudvQM8+49WXlzVzlOv\nbNxqXcCDow6YyWEyg+k1Yea01BKtLu1CXjn9YUF5xVtOsYLFW0iWFIZRqkkBIJ6Ms+SRi0hmkgCc\nfdBi5s88bIy9IJPJ8sbGbpa/0cHyNzt5ZnnrNtsEAh6Hz21m9xm1zG6uYdb0GDMao4RDwUm/joko\npz8sKK94yylWsHgLyZLCMEo5KQBks1keXvMoty//PQDhQJivHf15ZsZatusY6zfHaetN8uKKVh58\neg3J9MhPLh2813RmTo/R0hBhen2ExvpqptdFaKipIhCYmraKcvrDgvKKt5xiBYu3kCwpDKPUk8Kg\njfFNXPTkj0llXGd5b51zPKfsdRI14di4j5Efbzabpb07wRsbe1jfFmf95jiPLlu3TdvEcEJBjyP2\na2F6XYTGumqm11fTUFtNfSxMfU0V1eHgDjd0l9MfFpRXvOUUK1i8hWRJYRjlkhTA3cifa13Gdctu\nAcDD48x938Pxuy6kKhgec//xxhvvT7GhPU5bZz+buxNs7uqnvTvB34a0VYxHOBTg6P1nEI2EqImE\niVWHiA1OR9x0rNrNV4UDuWRSTn9YUF7xllOsYPEWUlklBREJANcA84AEcI6qrshb/wXgHGCwwvyT\nwD9G22c45ZQUBiUzKR5+8xHuXHFPbtn79n0Px88+hkho5PcVJivedCZDZ88Am7sTtPtJo6t3gK7e\nAZ58ZSPaKhZGAAATYUlEQVTJ1OS9VHfEfi1Uh4NEqoJUVwWJhN3v6qqgW543n5sOBwmHAoSCAYIB\nb0oez62UG0ExWLyFU4ikUMhHWM4AIqq6UEQWAJcCp+etnw/8m6o+PbhARM4cY5+dQjgQ4qTdT2TB\nrCP55mMX05/u584V93Dnins4cc6xnLDrscyqmVGw8wcDAabXu/aGoT5+6oG56Ww2S18iRU9/ir7+\nFL39SeL9KeKJvGl//omXNgx7ruEaySfb/rtPIxQKEA66RBIKBgiHvLzpIb+D/rq8fZo29xHv6c/t\nEwq57d16L7f/VCUpY4qlkCWFHwNPquqt/vwaVd01b/3LwIvALOAeVf3+WPsMJ5VKZ0Ml8sTNRPUM\n9HL/ir9w6wt35ZaFAyE+ceRiFsw5nEi4/N52zmazJFMZ+gfS9CdS9A2k6E+k6E+kc9N9A2kSAyn6\nEltv8+jza0kkd94uQI46cCYBzyMQcD/B/Gn/d/763LTHBPYZ7jzui8GW9Yy4f9Cf9jz3AmYg4E+z\nZZkH4C8L+K/jDC73/H3BncNtO7hu6/0DuW23PvbgtNli8L69g5/LlFcfXQ/coar3+vNvAHurasqf\n/yZwNdAF3An8N650MeI+wynH6qORpDNpntv0Ij9/6dZcgzTAMbPms2CX+SycO4+2Tb1FjHD8pvKz\nzWSyJNMZUukMqVTGn87mppMpf106QzKV3TLtb59KZ6mKhOjs7M8t++vz62xoVVPyrltyEsEJ9qFW\njOqjLiD/ba1AXkLwgMtVtdOfvwc4fLR9KkEwEOSIGYdyxIxD2dTXxhPrn+H/Vv6JJ9Y/zRPrn+aK\nv8M79ngrhzYfyB71uw37lnQlCgQ8qgOuLWKihiaxxSfvNxmhAZDJZslksmSzWTIZf95flsmSW5fO\nbL08Ozifv18mS31DlPbNvWSy5PbZsi3+/tnc7+w2y7bdJnfOrZZtiTXrr8tm3TKykHX/4W+6kYGd\nuGRXympjVfT19E/qMQuZFB4BTgNu89sHXshbVw8sE5EDgF7gbcCNQHSUfSpKc7SJ9+x1Mqfs+XZe\n7VjJ4+uf5vF1T3H/6w9x/+sPAbBwl6M4pPlA9p8+Nzc0qCk9Ac8jEJy86o+Wljpa66sn7Xg7Kr8d\najilViIfSznFWxsNl1VSuBM4WUQexdVdnSUii4FaVf2ZiCwBHsI9ZfSAqv6f/8TSVvsUML6yEPAC\nzG3ch7mN+/CpYz/M0uV/54VNL/HYur/lfgAObjqAg5sPYL/GfZgRbbY6WGPMhNh7CmXyjQC2jjeT\nzfB612qe3/RSruQwyMPj8BmHMHfa3uw7bW9m1cyY8qqmcv5sS105xQoWbyGV2yOppoACXoC9GvZg\nr4Y9OH2fU9jU18ZLbcqKjpU8vfE5ntn4PM9sfD63/bzmg9h32l7s27g3c2pnW3uEMWZYlhR2Es3R\nJk6YcywnzDmWs7KLae3bxIqOlfyj4zWeXP8Mz216kec2vZjb/qCm/dm7YU92q9uV3epmU181cg+u\nxpjKYUlhJ+R5HjNiLcyItXDs7KP56IH/SltfOys6XmNFx0oeX/8UL7a9wottr2zZB4+DmoQ5dbu6\nRFE7m+mRRmubMKbCWFKoEE3RRpqi8zlml/l86IAP0JnoYlXXG6zuXsubPWt4YdPLLGt7hWVDEsXc\nxn3YrXZ2rkQxI9ZiVU/G7MQsKVSohup65rUczLyWg3PLugd6eLN7Lau717C6Zw3PbHye5e0rWN6+\ndfdTe9XvwW51s5lT55LFLjWzCAfsn5IxOwP7SzY5dVW1HNC0Hwc0uRe3Pg70pfpZ07POJYruNTy1\n4VlWdr3Oyq7Xt9p3RqyZmX6V1azYDIQ9qBqooTZcY1VQxpQRSwpmVNFQxD21NG0vAP7twH8hmU6y\nrneDX6JYy5qetazsfION8U3Ay25HvxbKw2PP+t2YGZvhkkZNCzNjLTRHm6x0YUwJsr9Ks93CwTC7\n189h9/o5Wy3vGehlQ7yVDfFWuulg5aY1vNj2Ciu73mBl1xvbHKc52sQsv3Qx0/+ZEZtBfVWtlS6M\nKRJLCmbS1FbVUFtVwz7T9tzqpZp0Js2m/s1s6N3IhngrG/3E8WrnKjb1tUFe4za40smMWAst0Saa\no000R6bTHJ1Oc7SJhup6a+g2poAsKZiCCwaCuZLAUL3JeK50sTHemkscr3et5vWu1dtsH/KCTI82\n+smiyU8WLmFMjzQSHWWQImPM2CwpmKKqCcfYu2EP9m7YY6vl6UyajkQnm/o2s6m/jU19m2nr25yb\nd+0X24qGokyPTGN6pNH/cdNN/rw1fBszOksKpiQFA0GaotNpik5H2Heb9X2pPjb1tdPW18am/s20\n9rWxua+dzf3ttMY3saZn3bDHDQfCeUljS/LYIzML+kI0VNePOiSqMTs7SwqmLEVDUXari7Jb3ext\n1mWzWXqTcTb3t+f9dGw1vSE+ZJjQl7ZMRoLVNFTX01BV734P/lTVM626wZ+uIxwMF/gqjZl6lhTM\nTsfzvFyj99AnpAb1pxJs7m+nPdHB5v4OUqF+1m7eRMdAJ52JLjoTXdsmjiFqQrGtEkZDdT3ThiSR\n+qo6goHyHi7WVBZLCqYiRULVzK6dxezaWcDwXRCnMim6BrpzSaIj0UXnQNeW+YEu1vVuYG3v+hHP\n4+ES1LSqehoGSxnV9f78lp/acI09VWVKgiUFY0YQCoRybQ6jGUgPuIQxJGkMTnckOlnds5bVPWtH\nPEbAC+RKFnVVNdRW1VIXrqWuKu/Hn2/MxCb7Uo3JsaRgzA6qClYxI9bMjFjziNtks1n60/1bShx+\n0shN+8ljTe86Ut1jD0teE47lksSWBFJDXVVdXgKpoa6qlkgwYk9cmXGzpGDMFPA8j2goSjQUZVbN\nzBG3c8kjQfdADz3JHroH8n78+QT9tPV00J3sYX1845jnDgVCuSSRXwKpDddQE66hJhzzp93vWDhq\nVVkVzJKCMSXEJY+Ie6ub4UseQ98W70nG6Un20DXQ7ZLJQA/dyV4/mXTTPdBLd7KHdb0bSHavGTsG\nPGKhKDVVMWpCNdT6v2uqYtQO/vYTSm04Rk24hlgoag3qO4mCJQURCQDXAPOABHCOqq7IW/9B4PNA\nCngB+JSqZkTkGaDL32ylqp5VqBiNKXfBQJCG6joaquvYlV1G3TabzZJID+RKHb3JXnqScXqTvfT6\nv3uScXoGeulNxekd6B3xJcHhRENRavNKHYOlkJmt02Eg6JdCYkRDUWKhCNFQlEio2kolJaaQJYUz\ngIiqLhSRBcClwOkAIhIFvgscoqpxEfkNcKqI3A94qrqogHEZU5E8zyMSqiYSqqaFpnHtk81m6Uv1\n05uM05PszSWQniG/85e39rVtfZBteyvZEhMuJle1FiHmV7ENlpbyE0g0nDftL4+EIpZUJlkhk8Lx\nwB8BVPVxETkyb10COFZV43lx9ONKFTE/OYSAJar6eAFjNMaMwvM8YuEosXB0uxJJfzqRSxTBWIa1\nm9roSfYST8bpS/XTl+onnuqjL9XnppN9tPW1syY9/Jvoo4kEq/MSSZRYeOvEMTSJ5BJPOEI0GLFq\nryG8bDZbkAOLyPXAHap6rz//BrC3qqaGbPcZ4N3+z8HAAuB6YC5wLyBD98mXSqWzoZD9TzVmZ5DJ\nZIin+ogP9NGb7KN3IE482Ufcn+5N9hH3f+dPxwfitMY3T+ickVA1NeEYsaooNeEosaoYsbCbrqmK\nEgvH/Gl/eVVsq+1C5ZtUhn0krZAlhS6gLm8+kH9z99scfgjsB7xfVbMishxYoapZYLmItAG7MEoB\ntL09PtKqMQ33wlIpK6d4yylWKK94yylWmFi8HtXUUk1tYBpU437GIZPNkEgniCf7/VJIH/FU/5YS\nyWDpJLll2eA2b3auI8v2f0muCoSHqd5ypZJIsJpIMOKq7YLVbln+dNBV51UHJ9a2siP/Flpa6oZd\nXsik8AhwGnCb36bwwpD11+Kqkc5Q1Yy/7GzgEOBTIjIbqAe2vzxpjKlIAS+Qqy6C0V86HE4mm6Gu\nsYrV61tz1Vr5CaU/l1j6t044yT66E91s6N04ocQCUB2syksWEapD1UT9+eq85BEJVlMdrKIxMo3m\n5sMndK7RFDIp3AmcLCKP4oopZ4nIYqAWeAo3BPBfgQdFBOAK4AbgZhFZCmSBs0erOjLGmMkU8ALE\nwtEx32IfiXvCK0Ffqp/+dIL+VD/9qcSW6XTCn9+yPjG4zF8fT/axub+DZCY55vl2n/mfxGiYUKwj\nKVhS8L/9nzdkcf4QWyOVlRYXJiJjjCks94RXZFK6X09n0lsnkVSCRDpBIj1AfzpBOBBit/rZtLX1\nTkLkW9jLa8YYU4KCgSA1gRg14ZH7ugoEJv9xXHvA1xhjTI4lBWOMMTmWFIwxxuRYUjDGGJNjScEY\nY0yOJQVjjDE5lhSMMcbkWFIwxhiTU7BeUo0xxpQfKykYY4zJsaRgjDEmx5KCMcaYHEsKxhhjciwp\nGGOMybGkYIwxJseSgjHGmJyKHGRHRALANcA83DjR56jqiuJG5YjIMcDFqrpIRPYFbsYNTboM+A9V\nzYjIJ4BPAingu6r6hyLEGQZuBPbEDav+XeClEo43CFwHiB/feUB/qcbrxzwDeBo42Y+llGN9Bujy\nZ1cCF1Ha8X4NeC9QhbsXPFyK8YrIx4CP+bMR4DDgeODyQsVaqSWFM4CIqi4EvgpcWuR4ABCRC4Dr\ncf/zAX4MXKiqb8GNc326iMwCPgscB7wT+L6IVBch3A8DbX5s7wKuKvF4TwNQ1eOAC3E3rZKN10+6\n1wJ9/qJSjjUCeKq6yP85q8TjXQQc68dxIrBbqcarqjcPfq64LwifBb5RyFgrNSkcD/wRQFUfB44s\nbjg5rwJn5s3Px32DAbgXOAk4GnhEVROq2gmsAA6d0iid24Gv+9Me7ttJycarqv8LnOvP7gF0UMLx\nApcAPwXW+vOlHOs8ICYi94vIgyKyoMTjfSfwAnAncDfwB0o7XkTkSOAgVf1ZoWOt1KRQD3TmzadF\npOhVaap6B5DMW+Sp6mA/JN1AA9vGPrh8Sqlqj6p2i0gd8Dvct++SjRdAVVMi8nPgJ8CvKNF4/SqD\nVlW9L29xScbqi+OS2Dtx1XIl+9n6mnFfBP+JLfEGSjhegCXAt/3pgn62lZoUuoC6vPmAqqaKFcwo\nMnnTdbhvt0NjH1w+5URkN+Ah4BZV/TUlHi+Aqn4U2A/XvhDNW1VK8Z4NnCwif8bVIf8CmDFMTKUQ\nK8By4JeqmlXV5UAbMHOYuEol3jbgPlUdUFXFtS3l30BLKl4RmQaIqj7kLyro31mlJoVHgHcD+EXd\nF4obzoj+7td/ApwC/BV4EniLiEREpAE4ANfYNKVEZCZwP/AVVb2xDOL9iN+4CO6bbQZ4qhTjVdUT\nVPVEvx75WeDfgHtLMVbf2fjtciIyG/et9f4Sjncp8C4R8fx4a4AHSjjeE4AH8uYL+ndW9CqTIrkT\n903sUVx9+FlFjmckXwSuE5Eq4GXgd6qaFpErcf8QAsB/qmp/EWJbAjQCXxeRwbaFzwFXlmi8/wPc\nJCJ/AcLA5/0YS/XzHaqU/y3cANwsIktxT8ScDWwq1XhV9Q8icgLuRhoA/gP3xFRJxot7Yu61vPmC\n/luwrrONMcbkVGr1kTHGmGFYUjDGGJNjScEYY0yOJQVjjDE5lhSMMcbkWFIwZoJE5GgRudiffq+I\nfGcyj2lMMVTqewrGTIYD8d/cVdW7gLsm85jGFIO9p2B2av6bn0twbzEfgHt7fbGqDoyw/buA7+Be\ncFsJfEJV20TkElwX1mng98AVwPNALe5t3jXAIlX9mIisAn4LnIrrKHAJ7oWjucAXVfU2ETkY1wdT\nLa4Li0tx3VnkH/P7uC6S3457KewWVb3Yv6YfAkHcW6u/8OezQDvwQVXdtGOfnKlUVn1kKsGxwKdx\nSWF3XMdt2xCRFuAHwDtV9XDgPuBiEdkDOEVV5/nHmovrL+cbwF2qetEwh1urqgcBz+C6Z38Hrrvx\nwa42zsH1eX8U8FbgIlXtGHLM83DdOh+K6wXz/SLyHn///YC3+X05XQicp6pH4nr9PGICn5ExgCUF\nUxmWqeqbqprBdQswfYTtjsEljYdE5FlcIpmLKwX0icgjwBdwfdmP1YXAvf7v14GH/Q4XX8d1DQKu\n5BDx+2O6CFc6GOptwM2qmlbVOK43z7f769TvIhlctdWdInIV8LKq3j9GbMaMyJKCqQT5N/Asrr+r\n4QSBpap6mKoeBhwFfMC/oR+DGz+iCXhMRPYb45z51VPD9cB7G/A+3Gh1S0Y4xtC/T48t7YCDg++g\nqpcBi3B96P9QRP5zjNiMGZElBWO2eAJYmHfD/zrwIxE5HDeoyV9U9Uu4G7ngbvYTfVjjZOAbqvp7\n3Ohfg0OG5h/zQeCjIhIUkRjwIVxX5VsRkSeAOlW9HLgMqz4yO8CSgjE+VV2P6+HzNhF5AXdz/aKq\n/h14DFjmj0W8Clc99CSwQER+MIHTfQtY6h/vnf4x9xpyzGuBN4HngL/j2hruHOZYS3C9lD6NG13u\nmxOIxxjAnj4yxhiTx95TMBVFRKK4b/3D+Yb/voExFctKCsYYY3KsTcEYY0yOJQVjjDE5lhSMMcbk\nWFIwxhiTY0nBGGNMzv8HfLg+dCgzkGEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x116da3c90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#调整max_depth和min_child_weight之后再次调整n_estimators(6,4)\n",
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=699,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb2_3, X_train, y_train, cv_folds = kfold)\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#results = cross_val_score(xgb2_3, X_train, y_train, metrics='mlogloss', cv=kfold)\n",
    "#print results\n",
    "#print(\"CV logloss: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "IOError",
     "evalue": "File my_preds4_2_3_699.csv does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIOError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-d5ae43e0f42a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcvresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'my_preds4_2_3_699.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mcvresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcvresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# plot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtest_means\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcvresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'test-mlogloss-mean'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36mfrom_csv\u001b[0;34m(cls, path, header, sep, index_col, parse_dates, encoding, tupleize_cols, infer_datetime_format)\u001b[0m\n\u001b[1;32m   1229\u001b[0m                           \u001b[0mparse_dates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_dates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1230\u001b[0m                           \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtupleize_cols\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtupleize_cols\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1231\u001b[0;31m                           infer_datetime_format=infer_datetime_format)\n\u001b[0m\u001b[1;32m   1232\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1233\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mto_sparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'block'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda/lib/python2.7/site-packages/pandas/io/parsers.pyc\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, skip_footer, doublequote, delim_whitespace, as_recarray, compact_ints, use_unsigned, low_memory, buffer_lines, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    644\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    645\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 646\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    647\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda/lib/python2.7/site-packages/pandas/io/parsers.pyc\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    387\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    388\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 389\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    390\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    391\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mchunksize\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda/lib/python2.7/site-packages/pandas/io/parsers.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    728\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    729\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 730\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    731\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    732\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda/lib/python2.7/site-packages/pandas/io/parsers.pyc\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m    921\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    922\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 923\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    924\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    925\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda/lib/python2.7/site-packages/pandas/io/parsers.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1388\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'allow_leading_cols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex_col\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1389\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1390\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_parser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1391\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1392\u001b[0m         \u001b[0;31m# XXX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/parser.pyx\u001b[0m in \u001b[0;36mpandas.parser.TextReader.__cinit__ (pandas/parser.c:4184)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/parser.pyx\u001b[0m in \u001b[0;36mpandas.parser.TextReader._setup_parser_source (pandas/parser.c:8449)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mIOError\u001b[0m: File my_preds4_2_3_699.csv does not exist"
     ]
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('my_preds4_2_3_699.csv')\n",
    "\n",
    "cvresult = cvresult.iloc[100:]\n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(100,cvresult.shape[0]+100)\n",
    "        \n",
    "fig = pyplot.figure(figsize=(10, 10), dpi=100)\n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators_detail4_2_3_699.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
